#!/usr/bin/env python
# -*- coding:utf-8 -*-
import datetime
import multiprocessing
import six
import os
import sys

import six

from const import START_ARGS
from util import init_logger

if six.PY2:
    import Queue
else:
    import queue

__all__ = ['Service']


class Service:
    def __init__(self, conf, debug=False, mask='crawler-default'):
        self.start_time = datetime.datetime.now()
        self.init = False
        self.mask = mask
        self.workspace = conf.get('workspace', None)
        self.log, self.log_sh, _ = init_logger(name='crawler.core', sub_name='core', workspace=self.workspace,
                                               multiproc=True, stream=not conf.get('daemon'), debug=debug)
        try:
            # 配置项
            self.parsers = conf.get('parsers', [])

            self.daemon = conf.get('daemon')
            self.timeout = conf.get('timeout', 10)
            self.max_worker = conf.get('max_worker', 4)
            self.dbs_conf = conf.get('db', [])
            self.options = {k: v for k, v in six.iteritems(conf.get('options'))}
            self.msg_que = None
            self.running = None
            self.mm = None
            self.dbs = None
            self.init = True

            self.log.info('service|inited, good luck')

        except Exception:
            self.log.error('service|init failed', exc_info=1)
            sys.exit(1)

    def start(self):
        """
        启动服务

        :return:
        """
        try:
            if not self.init:
                raise Exception('service is not initialized')
            self.mm = multiprocessing.Manager()
            self.options = self.mm.dict(self.options)
            self.msg_que = self.mm.Queue()
            self.dbs = dict()
            self.db_init()
            for start_parser in self.parsers:
                if start_parser.get('status', False) is True:
                    self.msg_que.put((
                        start_parser.get('parser'),
                        START_ARGS
                    ))
            self.multi_master()
        except Exception:
            self.log.error('service|started failed', exc_info=1)
            sys.exit(1)
        finally:
            if self.dbs:
                self.db_close()

    def multi_master(self):
        if self.msg_que is None or self.mm is None:
            raise Exception('service|muti_master check msg_que or mm is None')
        self.running = set()
        while True:
            if self.msg_que.empty() and len(self.running) == 0:
                break

            while not self.msg_que.empty() and len(self.running) < self.max_worker:
                try:
                    msg = self.msg_que.get_nowait()
                except queue.Empty:
                    self.log.debug('multi_master|queue get warning', exc_info=1)
                    msg = None
                if not msg:
                    break
                proc = multiprocessing.Process(target=self.multi_worker, args=msg,
                                               kwargs={'que': self.msg_que, 'opt': self.options})
                proc.start()
                self.running.add(proc)

            remove_set = set()
            for proc in self.running:
                if not proc.is_alive():
                    remove_set.add(proc)
            for proc in remove_set:
                proc.join()
                self.running.remove(proc)
        return

    def multi_worker(self, parser, args, que, opt):
        parser_mod = __import__('parser.%s' % (parser,), fromlist=['Parser'])
        cr = parser_mod.Parser(name=parser, workspace=self.workspace, options=opt,
                               msg_que=que, stream=not self.daemon, timeout=self.timeout,
                               dbs=self.dbs)
        cr.log.info('worker|started|pid->%d', os.getpid())
        try:
            cr.start(args)
        except:
            cr.log.error('worker|error', exc_info=1)
        cr.log.info('worker|exited|pid->%d', os.getpid())

    def db_init(self):
        for dbc in self.dbs_conf:
            method = dbc.get('method')
            db_mod = __import__('db.%s' % (method,), fromlist=['Database'])
            db = db_mod.Database(**dbc)
            db.init()
            self.dbs[dbc.get('name')] = db

    def db_close(self):
        for _, db in six.iteritems(self.dbs):
            db.close()
